package com.lmq

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.dstream.DStream

object WordCount {
  def main(args: Array[String]): Unit = {
    /** DataFrame operations inside your streaming program */
    val words: DStream[String] = ???
    words.foreachRDD { rdd: RDD[String] =>
      // get the singleton instance of sparkSession
      val spark = SparkSession.builder.config(rdd.sparkContext.getConf).getOrCreate()
      import spark.implicits._

      // Convert RDD[String] to DataFrame
      val wordsDataFrame = rdd.toDF("word")
      // create a temporary view
      wordsDataFrame.createOrReplaceTempView("words")
      // do word count on DataFrame using SQL and print it
      val wordCountsDataFrame = spark.sql(
        """select word, count(*) as total
          |from from words group by word""".stripMargin)
      wordCountsDataFrame.show()


    }

  }

}
